{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "4f938ff6-f978-4344-bbab-593438fd7656",
   "metadata": {},
   "source": [
    "# Context Entitites Recall Evaluation\n",
    "\n",
    "Using Langchain, OpenAI, and Ragas"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "008eb19b-c6c6-42de-825a-6ca1d7598643",
   "metadata": {},
   "outputs": [],
   "source": [
    "!pip3 install ragas==0.1.13 datasets==2.20.0 langchain==0.2.12 openai==1.39.0 faiss-cpu==1.8.0.post1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1a033479-1155-4ddc-baf8-900cbbb966b1",
   "metadata": {},
   "outputs": [],
   "source": [
    "from ipython_secrets import get_secret \n",
    "import os\n",
    "\n",
    "os.environ['OPENAI_API_KEY'] = get_secret('OPENAI_API_KEY')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "babb22f7-943f-4aa8-b4d9-4b7e251df265",
   "metadata": {},
   "outputs": [],
   "source": [
    "from ragas.metrics import context_entity_recall\n",
    "from ragas import evaluate, RunConfig\n",
    "from datasets import load_dataset, Dataset\n",
    "from langchain_openai import ChatOpenAI, OpenAIEmbeddings\n",
    "from langchain_text_splitters import RecursiveCharacterTextSplitter\n",
    "from langchain_community.vectorstores import FAISS\n",
    "import os\n",
    "from typing import List\n",
    "\n",
    "# Add your OpenAI API key to the environment variables\n",
    "openai_api_key = os.getenv(\"OPENAI_API_KEY\")\n",
    "\n",
    "# Load sample dataset.\n",
    "dataset = load_dataset(\"explodinggradients/amnesty_qa\", split=\"eval\")\n",
    "\n",
    "sample_size = 100\n",
    "# Get sample questions from the sample dataset.\n",
    "sample_questions = dataset['question'][:sample_size]\n",
    "\n",
    "# Get sample context information from the sample dataset.\n",
    "sample_contexts = [item for row in dataset[\"contexts\"]\n",
    "                   [:sample_size] for item in row]\n",
    "\n",
    "sample_ground_truths = [item for row in dataset[\"ground_truths\"]\n",
    "                   [:sample_size] for item in row]\n",
    "\n",
    "# Break sample context into chunks to use with vector search.\n",
    "text_splitter = RecursiveCharacterTextSplitter(\n",
    "    chunk_size=400, chunk_overlap=100, add_start_index=True\n",
    ")\n",
    "chunks: List[str] = []\n",
    "for context in sample_contexts:\n",
    "    split_chunks = text_splitter.split_text(context)\n",
    "    chunks.extend(split_chunks)\n",
    "\n",
    "# Embedding models that we are evaluating.\n",
    "openai_embedding_models = [\"text-embedding-ada-002\", \"text-embedding-3-large\"]\n",
    "\n",
    "# Ragas evaluation config to use in all evaluations.\n",
    "ragas_run_config = RunConfig(max_workers=4, max_wait=180)\n",
    "\n",
    "# #Evaluate each embedding model\n",
    "for embedding_model in openai_embedding_models:\n",
    "\n",
    "    # Create an in-memory vector store for the evaluation.\n",
    "    db = FAISS.from_texts(\n",
    "        chunks, OpenAIEmbeddings(openai_api_key=openai_api_key, model=embedding_model))\n",
    "\n",
    "    # Get retrieved context using similarity search.\n",
    "    retrieval_contexts: List[str] = []\n",
    "    for question in sample_questions:\n",
    "        search_results = db.similarity_search(question)\n",
    "        retrieval_contexts.append(list(map(\n",
    "            lambda result: result.page_content, search_results)))\n",
    "\n",
    "    # Run evaluation for context relevancy of retrieved information.\n",
    "    result = evaluate(\n",
    "        dataset=Dataset.from_dict({\n",
    "            \"question\": sample_questions,\n",
    "            \"contexts\": retrieval_contexts,\n",
    "            \"ground_truth\": sample_ground_truths\n",
    "        }),\n",
    "        metrics=[context_entity_recall],\n",
    "        run_config=ragas_run_config,\n",
    "        raise_exceptions=False,\n",
    "        llm=ChatOpenAI(openai_api_key=openai_api_key, model_name=\"gpt-4o-mini\")\n",
    "    )\n",
    "    # Print out results\n",
    "    print(f\"Results for embedding model '{embedding_model}':\")\n",
    "    print(result)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "bf9e4d8b-eed7-44c3-879b-85ede465dd66",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
